#!/usr/bin/perl -w
# Proceses mcclist.com
# Just do a wget of the page mobile-network-codes-country-codes.

use strict;
use Text::Unidecode;
use HTML::Entities;


my $ifile = 'mcc_mnc.com';
my $ofile = 'mcc_mnc.com.tsv';

open(FD, "$ifile") or die "$!\n";
open(OFD, '>', $ofile) or die "$!\n";

my $mcc;
my $mnc;
my $operator_name;
my $network;
my $country;
my $state;

# oracle country dim read
open(OCD, 'oracle_country_dim.txt') or die;
my %ocd;
my @buf;

while (my $o = <OCD>) {
    chomp $o;
    if ($o =~ /^$/) {
        next;
    }
    if ($o =~ /^#/) {
	next;
    }

    my @oa = split(/\|/, $o);
    my $o_full_country = $oa[0];
    my $o_country2 = $oa[1];
    # for this mcc_mnc.com database, get rid of republic crap.
    my @o_full_breakup = split(/,/, $o_full_country);
    $o_full_country = $o_full_breakup[0];
    $o_country2 =~ s/\s+$//;
    $ocd{$o_full_country} = $o_country2;
}

while (my $line = <FD>) {
    chomp $line;
    $line = unidecode($line);
    $line = decode_entities($line);

    unless ($line =~ /<tr>/) {
        push @buf, $line;
    } else {
        while ($line !~ /<tr>.*<\/tr>/) {
            my $tmp = <FD>;
	    $tmp = unidecode($tmp);
	    $tmp = decode_entities($tmp);
            chomp $tmp;
            $line .= $tmp;
        }
        push @buf, $line;
    }
}
close FD;

for my $line (@buf) {

    next if ($line =~ /^$/);
    my @a = split(/<td/, $line);

    # country title header
    if (scalar(@a) == 1) {
	if ($a[0] =~ /<h3><a name=\"(.*)\"\s+href/) {
	    $country = $1;
	    if ($country =~ /Ivoire/) {
		$country = "Cote d'Ivoire";
	    } elsif ($country =~ /Hongkong/) {
		$country = 'Hong Kong';
	    } elsif ($country =~ /Saint Helena/) {
		$country = 'Saint Helena';
	    }
	    # lookup iso2 ltr
	    $country = $ocd{$country};
	    unless (defined $country) {
		$country = 'ZZ';
	    }
	    next;
	}
    }
    next if ($a[0] !~ /<tr>/);

    # table header
    if ($a[0] =~ />MCC</) {
	next;
    }
    
    # Contents
    if ($a[1] =~ />(\d+)<\/td>/) {
	$mcc = $1;
    } else {
	$mcc = "";
    }
    
    if ($a[2] =~ />(\d+)\W*<\/td>/) {
	$mnc = $1;
    } else {
	$mnc = "";
    }

    # network (optional, use if no operator name)
    if ($a[3] =~ />(.*)<\/td>/) {
	$network = $1;
    } else {
	$network = "";
    }

    if ($a[4] =~ />(.*)<\/td>/) {
	$operator_name = $1;
    } else {
	$operator_name = "";
    }
    $operator_name =~ s/\t//g;
    $network =~ s/\t//g;
    if ($operator_name eq "" and $network ne "") {
	$operator_name = $network;
    }

    if ($operator_name =~ /^\s*$/) {
        $operator_name = "unknown";
    }
    # try to covert diacritic chars to ascii in operator_name.
    $operator_name =~ s/A3/o/g;
    $operator_name =~ s/A\+\-/n/g;
    $operator_name =~ s/A\!/s/g;
    $operator_name =~ s/A1\/2/Z/g;
    $operator_name =~ s/AmA\(c\)/A/g;
    $operator_name =~ s/Ao/u/g;
    $operator_name =~ s/\[|\]//g;

    
    if ($a[5] =~ />(.*)<\/td>.*<\/tr>/) {
	$state = $1;
    } else {
	$state = "";
    }

    unless ($mcc eq "" and $mnc eq "" and $operator_name eq "") {
	print OFD "$mcc\t$mnc\t$operator_name\t$country\n";
    }
}

close OFD;
